{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "        android_id  apptype  carrier  dev_height  dev_ppi  dev_width  label  \\\n0           316361     1199  46000.0         0.0      0.0        0.0      1   \n1           135939      893      0.0         0.0      0.0        0.0      1   \n2           399254      821      0.0       760.0      0.0      360.0      1   \n3            68983     1004  46000.0      2214.0      0.0     1080.0      0   \n4           288999     1076  46000.0      2280.0      0.0     1080.0      1   \n...            ...      ...      ...         ...      ...        ...    ...   \n499995      392477     1028  46000.0      1920.0      3.0     1080.0      1   \n499996      346134     1001      0.0      1424.0      0.0      720.0      0   \n499997      499635      761  46000.0      1280.0      0.0      720.0      0   \n499998      239786      917  46001.0       960.0      0.0      540.0      0   \n499999      270531      929  46000.0      2040.0      3.0     1080.0      1   \n\n          lan  media_id  ntt       os    osv  package      sid     timestamp  \\\n0         NaN       104  6.0  android      9       18  1438873  1.559893e+12   \n1         NaN        19  6.0  android    8.1        0  1185582  1.559994e+12   \n2         NaN       559  0.0  android  8.1.0        0  1555716  1.559837e+12   \n3         NaN       129  2.0  android  8.1.0        0  1093419  1.560042e+12   \n4       zh-CN        64  2.0  android  8.0.0        0  1400089  1.559867e+12   \n...       ...       ...  ...      ...    ...      ...      ...           ...   \n499995  zh-CN       144  6.0  Android  7.1.2       25  1546078  1.559834e+12   \n499996    NaN        29  2.0  android  8.1.0        0  1480612  1.559814e+12   \n499997    NaN        54  6.0  android  6.0.1        9  1698442  1.559676e+12   \n499998  zh_CN       109  2.0  android  5.1.1        0  1331155  1.559840e+12   \n499999  zh-CN        59  2.0  Android  8.1.0       78  1373973  1.559922e+12   \n\n       version    fea_hash  location   fea1_hash  cus_type  \n0            8  2135019403         0  2329670524       601  \n1            4  2782306428         1  2864801071      1000  \n2            0  1392806005         2   628911675       696  \n3            0  3562553457         3  1283809327       753  \n4            5  2364522023         4  1510695983       582  \n...        ...         ...       ...         ...       ...  \n499995       7   861755946        79   140647032       373  \n499996       3  1714444511        23  2745131047       525  \n499997       0  3843262581        25  1326115882       810  \n499998       0  1984296118       225  1446741112       772  \n499999       5  1697301943        49  1915763579      1076  \n\n[500000 rows x 20 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>android_id</th>\n      <th>apptype</th>\n      <th>carrier</th>\n      <th>dev_height</th>\n      <th>dev_ppi</th>\n      <th>dev_width</th>\n      <th>label</th>\n      <th>lan</th>\n      <th>media_id</th>\n      <th>ntt</th>\n      <th>os</th>\n      <th>osv</th>\n      <th>package</th>\n      <th>sid</th>\n      <th>timestamp</th>\n      <th>version</th>\n      <th>fea_hash</th>\n      <th>location</th>\n      <th>fea1_hash</th>\n      <th>cus_type</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>316361</td>\n      <td>1199</td>\n      <td>46000.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1</td>\n      <td>NaN</td>\n      <td>104</td>\n      <td>6.0</td>\n      <td>android</td>\n      <td>9</td>\n      <td>18</td>\n      <td>1438873</td>\n      <td>1.559893e+12</td>\n      <td>8</td>\n      <td>2135019403</td>\n      <td>0</td>\n      <td>2329670524</td>\n      <td>601</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>135939</td>\n      <td>893</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1</td>\n      <td>NaN</td>\n      <td>19</td>\n      <td>6.0</td>\n      <td>android</td>\n      <td>8.1</td>\n      <td>0</td>\n      <td>1185582</td>\n      <td>1.559994e+12</td>\n      <td>4</td>\n      <td>2782306428</td>\n      <td>1</td>\n      <td>2864801071</td>\n      <td>1000</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>399254</td>\n      <td>821</td>\n      <td>0.0</td>\n      <td>760.0</td>\n      <td>0.0</td>\n      <td>360.0</td>\n      <td>1</td>\n      <td>NaN</td>\n      <td>559</td>\n      <td>0.0</td>\n      <td>android</td>\n      <td>8.1.0</td>\n      <td>0</td>\n      <td>1555716</td>\n      <td>1.559837e+12</td>\n      <td>0</td>\n      <td>1392806005</td>\n      <td>2</td>\n      <td>628911675</td>\n      <td>696</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>68983</td>\n      <td>1004</td>\n      <td>46000.0</td>\n      <td>2214.0</td>\n      <td>0.0</td>\n      <td>1080.0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>129</td>\n      <td>2.0</td>\n      <td>android</td>\n      <td>8.1.0</td>\n      <td>0</td>\n      <td>1093419</td>\n      <td>1.560042e+12</td>\n      <td>0</td>\n      <td>3562553457</td>\n      <td>3</td>\n      <td>1283809327</td>\n      <td>753</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>288999</td>\n      <td>1076</td>\n      <td>46000.0</td>\n      <td>2280.0</td>\n      <td>0.0</td>\n      <td>1080.0</td>\n      <td>1</td>\n      <td>zh-CN</td>\n      <td>64</td>\n      <td>2.0</td>\n      <td>android</td>\n      <td>8.0.0</td>\n      <td>0</td>\n      <td>1400089</td>\n      <td>1.559867e+12</td>\n      <td>5</td>\n      <td>2364522023</td>\n      <td>4</td>\n      <td>1510695983</td>\n      <td>582</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>499995</th>\n      <td>392477</td>\n      <td>1028</td>\n      <td>46000.0</td>\n      <td>1920.0</td>\n      <td>3.0</td>\n      <td>1080.0</td>\n      <td>1</td>\n      <td>zh-CN</td>\n      <td>144</td>\n      <td>6.0</td>\n      <td>Android</td>\n      <td>7.1.2</td>\n      <td>25</td>\n      <td>1546078</td>\n      <td>1.559834e+12</td>\n      <td>7</td>\n      <td>861755946</td>\n      <td>79</td>\n      <td>140647032</td>\n      <td>373</td>\n    </tr>\n    <tr>\n      <th>499996</th>\n      <td>346134</td>\n      <td>1001</td>\n      <td>0.0</td>\n      <td>1424.0</td>\n      <td>0.0</td>\n      <td>720.0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>29</td>\n      <td>2.0</td>\n      <td>android</td>\n      <td>8.1.0</td>\n      <td>0</td>\n      <td>1480612</td>\n      <td>1.559814e+12</td>\n      <td>3</td>\n      <td>1714444511</td>\n      <td>23</td>\n      <td>2745131047</td>\n      <td>525</td>\n    </tr>\n    <tr>\n      <th>499997</th>\n      <td>499635</td>\n      <td>761</td>\n      <td>46000.0</td>\n      <td>1280.0</td>\n      <td>0.0</td>\n      <td>720.0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>54</td>\n      <td>6.0</td>\n      <td>android</td>\n      <td>6.0.1</td>\n      <td>9</td>\n      <td>1698442</td>\n      <td>1.559676e+12</td>\n      <td>0</td>\n      <td>3843262581</td>\n      <td>25</td>\n      <td>1326115882</td>\n      <td>810</td>\n    </tr>\n    <tr>\n      <th>499998</th>\n      <td>239786</td>\n      <td>917</td>\n      <td>46001.0</td>\n      <td>960.0</td>\n      <td>0.0</td>\n      <td>540.0</td>\n      <td>0</td>\n      <td>zh_CN</td>\n      <td>109</td>\n      <td>2.0</td>\n      <td>android</td>\n      <td>5.1.1</td>\n      <td>0</td>\n      <td>1331155</td>\n      <td>1.559840e+12</td>\n      <td>0</td>\n      <td>1984296118</td>\n      <td>225</td>\n      <td>1446741112</td>\n      <td>772</td>\n    </tr>\n    <tr>\n      <th>499999</th>\n      <td>270531</td>\n      <td>929</td>\n      <td>46000.0</td>\n      <td>2040.0</td>\n      <td>3.0</td>\n      <td>1080.0</td>\n      <td>1</td>\n      <td>zh-CN</td>\n      <td>59</td>\n      <td>2.0</td>\n      <td>Android</td>\n      <td>8.1.0</td>\n      <td>78</td>\n      <td>1373973</td>\n      <td>1.559922e+12</td>\n      <td>5</td>\n      <td>1697301943</td>\n      <td>49</td>\n      <td>1915763579</td>\n      <td>1076</td>\n    </tr>\n  </tbody>\n</table>\n<p>500000 rows × 20 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_path = r'datasets/train.csv'\n",
    "data = pd.read_csv(data_path)\n",
    "data = data.iloc[:,1:]\n",
    "data"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "Index(['lan', 'os', 'osv', 'version', 'fea_hash'], dtype='object')"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols = data.columns\n",
    "object_cols = data.select_dtypes(include='object').columns\n",
    "object_cols"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "lan    183280\nosv      6561\ndtype: int64"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 缺失值个数\n",
    "temp = data.isnull().sum()\n",
    "# 有缺失值的字段： lan, osv\n",
    "temp[temp>0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['android_id', 'apptype', 'carrier', 'dev_height', 'dev_ppi', 'dev_width', 'lan', 'media_id', 'ntt', 'os', 'osv', 'package', 'sid', 'timestamp', 'version', 'fea_hash', 'location', 'fea1_hash', 'cus_type']\n"
     ]
    }
   ],
   "source": [
    "features = data.columns.tolist()\n",
    "label = data['label']\n",
    "features.remove('label')\n",
    "print(features)\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "android_id 362258\n",
      "apptype 89\n",
      "carrier 5\n",
      "dev_height 798\n",
      "dev_ppi 92\n",
      "dev_width 346\n",
      "lan 21\n",
      "media_id 284\n",
      "ntt 8\n",
      "os 2\n",
      "osv 154\n",
      "package 1950\n",
      "sid 500000\n",
      "timestamp 500000\n",
      "version 22\n",
      "fea_hash 402980\n",
      "location 332\n",
      "fea1_hash 4959\n",
      "cus_type 58\n"
     ]
    }
   ],
   "source": [
    "#统计不唯一的值的个数\n",
    "for feature in features:\n",
    "    print(feature, data[feature].nunique())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "10    391669\n9      99347\n8       8977\n7          6\n5          1\nName: fea1_hash, dtype: int64"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['fea_hash'].map(lambda x: len(str(x))).value_counts()\n",
    "data['fea1_hash'].map(lambda x: len(str(x))).value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "['android_id',\n 'apptype',\n 'carrier',\n 'dev_height',\n 'dev_ppi',\n 'dev_width',\n 'lan',\n 'media_id',\n 'ntt',\n 'package',\n 'timestamp',\n 'version',\n 'fea_hash',\n 'location',\n 'fea1_hash',\n 'cus_type']"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_list = ['os', 'osv', 'sid']\n",
    "col = features\n",
    "for i in remove_list:\n",
    "    col.remove(i)\n",
    "col\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-9-d2e78b4b5e93>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  features['fea_hash_len'] = features['fea_hash'].map(lambda x: len(str(x)))\n",
      "<ipython-input-9-d2e78b4b5e93>:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  features['fea1_hash_len'] = features['fea1_hash'].map(lambda x: len(str(x)))\n",
      "<ipython-input-9-d2e78b4b5e93>:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  features['fea_hash'] = features['fea_hash'].map(lambda x: 0 if len(str(x))>16 else int(x))\n",
      "<ipython-input-9-d2e78b4b5e93>:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  features['fea1_hash'] = features['fea1_hash'].map(lambda x: 0 if len(str(x))>16 else int(x))\n"
     ]
    },
    {
     "data": {
      "text/plain": "        android_id  apptype  carrier  dev_height  dev_ppi  dev_width    lan  \\\n0           316361     1199  46000.0         0.0      0.0        0.0    NaN   \n1           135939      893      0.0         0.0      0.0        0.0    NaN   \n2           399254      821      0.0       760.0      0.0      360.0    NaN   \n3            68983     1004  46000.0      2214.0      0.0     1080.0    NaN   \n4           288999     1076  46000.0      2280.0      0.0     1080.0  zh-CN   \n...            ...      ...      ...         ...      ...        ...    ...   \n499995      392477     1028  46000.0      1920.0      3.0     1080.0  zh-CN   \n499996      346134     1001      0.0      1424.0      0.0      720.0    NaN   \n499997      499635      761  46000.0      1280.0      0.0      720.0    NaN   \n499998      239786      917  46001.0       960.0      0.0      540.0  zh_CN   \n499999      270531      929  46000.0      2040.0      3.0     1080.0  zh-CN   \n\n        media_id  ntt  package     timestamp version    fea_hash  location  \\\n0            104  6.0       18  1.559893e+12       8  2135019403         0   \n1             19  6.0        0  1.559994e+12       4  2782306428         1   \n2            559  0.0        0  1.559837e+12       0  1392806005         2   \n3            129  2.0        0  1.560042e+12       0  3562553457         3   \n4             64  2.0        0  1.559867e+12       5  2364522023         4   \n...          ...  ...      ...           ...     ...         ...       ...   \n499995       144  6.0       25  1.559834e+12       7   861755946        79   \n499996        29  2.0        0  1.559814e+12       3  1714444511        23   \n499997        54  6.0        9  1.559676e+12       0  3843262581        25   \n499998       109  2.0        0  1.559840e+12       0  1984296118       225   \n499999        59  2.0       78  1.559922e+12       5  1697301943        49   \n\n         fea1_hash  cus_type  fea_hash_len  fea1_hash_len  \n0       2329670524       601            10             10  \n1       2864801071      1000            10             10  \n2        628911675       696            10              9  \n3       1283809327       753            10             10  \n4       1510695983       582            10             10  \n...            ...       ...           ...            ...  \n499995   140647032       373             9              9  \n499996  2745131047       525            10             10  \n499997  1326115882       810            10             10  \n499998  1446741112       772            10             10  \n499999  1915763579      1076            10             10  \n\n[500000 rows x 18 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>android_id</th>\n      <th>apptype</th>\n      <th>carrier</th>\n      <th>dev_height</th>\n      <th>dev_ppi</th>\n      <th>dev_width</th>\n      <th>lan</th>\n      <th>media_id</th>\n      <th>ntt</th>\n      <th>package</th>\n      <th>timestamp</th>\n      <th>version</th>\n      <th>fea_hash</th>\n      <th>location</th>\n      <th>fea1_hash</th>\n      <th>cus_type</th>\n      <th>fea_hash_len</th>\n      <th>fea1_hash_len</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>316361</td>\n      <td>1199</td>\n      <td>46000.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>NaN</td>\n      <td>104</td>\n      <td>6.0</td>\n      <td>18</td>\n      <td>1.559893e+12</td>\n      <td>8</td>\n      <td>2135019403</td>\n      <td>0</td>\n      <td>2329670524</td>\n      <td>601</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>135939</td>\n      <td>893</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>NaN</td>\n      <td>19</td>\n      <td>6.0</td>\n      <td>0</td>\n      <td>1.559994e+12</td>\n      <td>4</td>\n      <td>2782306428</td>\n      <td>1</td>\n      <td>2864801071</td>\n      <td>1000</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>399254</td>\n      <td>821</td>\n      <td>0.0</td>\n      <td>760.0</td>\n      <td>0.0</td>\n      <td>360.0</td>\n      <td>NaN</td>\n      <td>559</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>1.559837e+12</td>\n      <td>0</td>\n      <td>1392806005</td>\n      <td>2</td>\n      <td>628911675</td>\n      <td>696</td>\n      <td>10</td>\n      <td>9</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>68983</td>\n      <td>1004</td>\n      <td>46000.0</td>\n      <td>2214.0</td>\n      <td>0.0</td>\n      <td>1080.0</td>\n      <td>NaN</td>\n      <td>129</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>1.560042e+12</td>\n      <td>0</td>\n      <td>3562553457</td>\n      <td>3</td>\n      <td>1283809327</td>\n      <td>753</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>288999</td>\n      <td>1076</td>\n      <td>46000.0</td>\n      <td>2280.0</td>\n      <td>0.0</td>\n      <td>1080.0</td>\n      <td>zh-CN</td>\n      <td>64</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>1.559867e+12</td>\n      <td>5</td>\n      <td>2364522023</td>\n      <td>4</td>\n      <td>1510695983</td>\n      <td>582</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>499995</th>\n      <td>392477</td>\n      <td>1028</td>\n      <td>46000.0</td>\n      <td>1920.0</td>\n      <td>3.0</td>\n      <td>1080.0</td>\n      <td>zh-CN</td>\n      <td>144</td>\n      <td>6.0</td>\n      <td>25</td>\n      <td>1.559834e+12</td>\n      <td>7</td>\n      <td>861755946</td>\n      <td>79</td>\n      <td>140647032</td>\n      <td>373</td>\n      <td>9</td>\n      <td>9</td>\n    </tr>\n    <tr>\n      <th>499996</th>\n      <td>346134</td>\n      <td>1001</td>\n      <td>0.0</td>\n      <td>1424.0</td>\n      <td>0.0</td>\n      <td>720.0</td>\n      <td>NaN</td>\n      <td>29</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>1.559814e+12</td>\n      <td>3</td>\n      <td>1714444511</td>\n      <td>23</td>\n      <td>2745131047</td>\n      <td>525</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>499997</th>\n      <td>499635</td>\n      <td>761</td>\n      <td>46000.0</td>\n      <td>1280.0</td>\n      <td>0.0</td>\n      <td>720.0</td>\n      <td>NaN</td>\n      <td>54</td>\n      <td>6.0</td>\n      <td>9</td>\n      <td>1.559676e+12</td>\n      <td>0</td>\n      <td>3843262581</td>\n      <td>25</td>\n      <td>1326115882</td>\n      <td>810</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>499998</th>\n      <td>239786</td>\n      <td>917</td>\n      <td>46001.0</td>\n      <td>960.0</td>\n      <td>0.0</td>\n      <td>540.0</td>\n      <td>zh_CN</td>\n      <td>109</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>1.559840e+12</td>\n      <td>0</td>\n      <td>1984296118</td>\n      <td>225</td>\n      <td>1446741112</td>\n      <td>772</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n    <tr>\n      <th>499999</th>\n      <td>270531</td>\n      <td>929</td>\n      <td>46000.0</td>\n      <td>2040.0</td>\n      <td>3.0</td>\n      <td>1080.0</td>\n      <td>zh-CN</td>\n      <td>59</td>\n      <td>2.0</td>\n      <td>78</td>\n      <td>1.559922e+12</td>\n      <td>5</td>\n      <td>1697301943</td>\n      <td>49</td>\n      <td>1915763579</td>\n      <td>1076</td>\n      <td>10</td>\n      <td>10</td>\n    </tr>\n  </tbody>\n</table>\n<p>500000 rows × 18 columns</p>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 特征筛选\n",
    "features = data[col]\n",
    "# 构造fea_hash_len特征\n",
    "features['fea_hash_len'] = features['fea_hash'].map(lambda x: len(str(x)))\n",
    "features['fea1_hash_len'] = features['fea1_hash'].map(lambda x: len(str(x)))\n",
    "# Thinking：为什么将很大的，很长的fea_hash化为0？\n",
    "# 如果fea_hash很长，都归为0，否则为自己的本身\n",
    "features['fea_hash'] = features['fea_hash'].map(lambda x: 0 if len(str(x))>16 else int(x))\n",
    "features['fea1_hash'] = features['fea1_hash'].map(lambda x: 0 if len(str(x))>16 else int(x))\n",
    "features\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-10-72240e594f8e>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  features['year'] = temp.year\n"
     ]
    },
    {
     "data": {
      "text/plain": "        android_id  apptype  carrier  dev_height  dev_ppi  dev_width    lan  \\\n0           316361     1199  46000.0         0.0      0.0        0.0    NaN   \n1           135939      893      0.0         0.0      0.0        0.0    NaN   \n2           399254      821      0.0       760.0      0.0      360.0    NaN   \n3            68983     1004  46000.0      2214.0      0.0     1080.0    NaN   \n4           288999     1076  46000.0      2280.0      0.0     1080.0  zh-CN   \n...            ...      ...      ...         ...      ...        ...    ...   \n499995      392477     1028  46000.0      1920.0      3.0     1080.0  zh-CN   \n499996      346134     1001      0.0      1424.0      0.0      720.0    NaN   \n499997      499635      761  46000.0      1280.0      0.0      720.0    NaN   \n499998      239786      917  46001.0       960.0      0.0      540.0  zh_CN   \n499999      270531      929  46000.0      2040.0      3.0     1080.0  zh-CN   \n\n        media_id  ntt  package  ...   fea1_hash cus_type  fea_hash_len  \\\n0            104  6.0       18  ...  2329670524      601            10   \n1             19  6.0        0  ...  2864801071     1000            10   \n2            559  0.0        0  ...   628911675      696            10   \n3            129  2.0        0  ...  1283809327      753            10   \n4             64  2.0        0  ...  1510695983      582            10   \n...          ...  ...      ...  ...         ...      ...           ...   \n499995       144  6.0       25  ...   140647032      373             9   \n499996        29  2.0        0  ...  2745131047      525            10   \n499997        54  6.0        9  ...  1326115882      810            10   \n499998       109  2.0        0  ...  1446741112      772            10   \n499999        59  2.0       78  ...  1915763579     1076            10   \n\n        fea1_hash_len  year  month  day  week_day  hour  minute  \n0                  10  2019      6    7         4     7      32  \n1                  10  2019      6    8         5    11      40  \n2                   9  2019      6    6         3    15      58  \n3                  10  2019      6    9         6     0      59  \n4                  10  2019      6    7         4     0      28  \n...               ...   ...    ...  ...       ...   ...     ...  \n499995              9  2019      6    6         3    15      14  \n499996             10  2019      6    6         3     9      40  \n499997             10  2019      6    4         1    19      14  \n499998             10  2019      6    6         3    16      59  \n499999             10  2019      6    7         4    15      32  \n\n[500000 rows x 24 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>android_id</th>\n      <th>apptype</th>\n      <th>carrier</th>\n      <th>dev_height</th>\n      <th>dev_ppi</th>\n      <th>dev_width</th>\n      <th>lan</th>\n      <th>media_id</th>\n      <th>ntt</th>\n      <th>package</th>\n      <th>...</th>\n      <th>fea1_hash</th>\n      <th>cus_type</th>\n      <th>fea_hash_len</th>\n      <th>fea1_hash_len</th>\n      <th>year</th>\n      <th>month</th>\n      <th>day</th>\n      <th>week_day</th>\n      <th>hour</th>\n      <th>minute</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>316361</td>\n      <td>1199</td>\n      <td>46000.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>NaN</td>\n      <td>104</td>\n      <td>6.0</td>\n      <td>18</td>\n      <td>...</td>\n      <td>2329670524</td>\n      <td>601</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>7</td>\n      <td>4</td>\n      <td>7</td>\n      <td>32</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>135939</td>\n      <td>893</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>NaN</td>\n      <td>19</td>\n      <td>6.0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>2864801071</td>\n      <td>1000</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>8</td>\n      <td>5</td>\n      <td>11</td>\n      <td>40</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>399254</td>\n      <td>821</td>\n      <td>0.0</td>\n      <td>760.0</td>\n      <td>0.0</td>\n      <td>360.0</td>\n      <td>NaN</td>\n      <td>559</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>628911675</td>\n      <td>696</td>\n      <td>10</td>\n      <td>9</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>6</td>\n      <td>3</td>\n      <td>15</td>\n      <td>58</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>68983</td>\n      <td>1004</td>\n      <td>46000.0</td>\n      <td>2214.0</td>\n      <td>0.0</td>\n      <td>1080.0</td>\n      <td>NaN</td>\n      <td>129</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1283809327</td>\n      <td>753</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>9</td>\n      <td>6</td>\n      <td>0</td>\n      <td>59</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>288999</td>\n      <td>1076</td>\n      <td>46000.0</td>\n      <td>2280.0</td>\n      <td>0.0</td>\n      <td>1080.0</td>\n      <td>zh-CN</td>\n      <td>64</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1510695983</td>\n      <td>582</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>7</td>\n      <td>4</td>\n      <td>0</td>\n      <td>28</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>499995</th>\n      <td>392477</td>\n      <td>1028</td>\n      <td>46000.0</td>\n      <td>1920.0</td>\n      <td>3.0</td>\n      <td>1080.0</td>\n      <td>zh-CN</td>\n      <td>144</td>\n      <td>6.0</td>\n      <td>25</td>\n      <td>...</td>\n      <td>140647032</td>\n      <td>373</td>\n      <td>9</td>\n      <td>9</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>6</td>\n      <td>3</td>\n      <td>15</td>\n      <td>14</td>\n    </tr>\n    <tr>\n      <th>499996</th>\n      <td>346134</td>\n      <td>1001</td>\n      <td>0.0</td>\n      <td>1424.0</td>\n      <td>0.0</td>\n      <td>720.0</td>\n      <td>NaN</td>\n      <td>29</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>2745131047</td>\n      <td>525</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>6</td>\n      <td>3</td>\n      <td>9</td>\n      <td>40</td>\n    </tr>\n    <tr>\n      <th>499997</th>\n      <td>499635</td>\n      <td>761</td>\n      <td>46000.0</td>\n      <td>1280.0</td>\n      <td>0.0</td>\n      <td>720.0</td>\n      <td>NaN</td>\n      <td>54</td>\n      <td>6.0</td>\n      <td>9</td>\n      <td>...</td>\n      <td>1326115882</td>\n      <td>810</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>4</td>\n      <td>1</td>\n      <td>19</td>\n      <td>14</td>\n    </tr>\n    <tr>\n      <th>499998</th>\n      <td>239786</td>\n      <td>917</td>\n      <td>46001.0</td>\n      <td>960.0</td>\n      <td>0.0</td>\n      <td>540.0</td>\n      <td>zh_CN</td>\n      <td>109</td>\n      <td>2.0</td>\n      <td>0</td>\n      <td>...</td>\n      <td>1446741112</td>\n      <td>772</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>6</td>\n      <td>3</td>\n      <td>16</td>\n      <td>59</td>\n    </tr>\n    <tr>\n      <th>499999</th>\n      <td>270531</td>\n      <td>929</td>\n      <td>46000.0</td>\n      <td>2040.0</td>\n      <td>3.0</td>\n      <td>1080.0</td>\n      <td>zh-CN</td>\n      <td>59</td>\n      <td>2.0</td>\n      <td>78</td>\n      <td>...</td>\n      <td>1915763579</td>\n      <td>1076</td>\n      <td>10</td>\n      <td>10</td>\n      <td>2019</td>\n      <td>6</td>\n      <td>7</td>\n      <td>4</td>\n      <td>15</td>\n      <td>32</td>\n    </tr>\n  </tbody>\n</table>\n<p>500000 rows × 24 columns</p>\n</div>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "time = pd.to_datetime(features['timestamp'],unit='ms')\n",
    "temp = pd.DatetimeIndex(time)\n",
    "\n",
    "features['year'] = temp.year\n",
    "features['month'] = temp.month\n",
    "features['day'] = temp.day\n",
    "features['week_day'] = temp.weekday\n",
    "features['hour'] = temp.hour\n",
    "features['minute'] = temp.minute\n",
    "features"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "8.1.0                     151419\n9                          71332\n6.0.1                      31714\n7.1.1                      26503\n9.0.0                      24385\n                           ...  \n8.0.2                          1\nAndroid 7.1                    1\nAndroid 5.12                   1\n6.1.0-RS-20190305.1125         1\n9.1.0                          1\nName: osv, Length: 154, dtype: int64"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['osv'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [],
   "source": [
    "bare_osv=data['osv'].value_counts()[data['osv'].value_counts()<20].index\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "0    499724\n1       276\nName: osv_2, dtype: int64"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def f(x):\n",
    "    # 判断是否在关键特征值里，是1，否0\n",
    "    if x in bare_osv:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "data['osv_2']=data['osv'].apply(f,args=())\n",
    "data['osv_2'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "\u001B[1;32m<ipython-input-15-2603c27a8fad>\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      6\u001B[0m     \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      7\u001B[0m         \u001B[1;32mreturn\u001B[0m \u001B[1;36m0\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 8\u001B[1;33m \u001B[0mdata\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'osv_3'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mdata\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'osv'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mapply\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mf\u001B[0m\u001B[1;33m,\u001B[0m\u001B[0margs\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m      9\u001B[0m \u001B[0mdata\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;34m'osv_3'\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     10\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001B[0m in \u001B[0;36mapply\u001B[1;34m(self, func, convert_dtype, args, **kwds)\u001B[0m\n\u001B[0;32m   4136\u001B[0m             \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   4137\u001B[0m                 \u001B[0mvalues\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mastype\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mobject\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_values\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m-> 4138\u001B[1;33m                 \u001B[0mmapped\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mlib\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmap_infer\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mvalues\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mf\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mconvert\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mconvert_dtype\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   4139\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   4140\u001B[0m         \u001B[1;32mif\u001B[0m \u001B[0mlen\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mmapped\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mand\u001B[0m \u001B[0misinstance\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mmapped\u001B[0m\u001B[1;33m[\u001B[0m\u001B[1;36m0\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mSeries\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mpandas\\_libs\\lib.pyx\u001B[0m in \u001B[0;36mpandas._libs.lib.map_infer\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;32m<ipython-input-15-2603c27a8fad>\u001B[0m in \u001B[0;36mf\u001B[1;34m(x)\u001B[0m\n\u001B[0;32m      3\u001B[0m     \u001B[1;31m# 判断是否在关键特征值里，是1，否0\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      4\u001B[0m     \u001B[1;32mif\u001B[0m \u001B[0mx\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mosv_counts\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 5\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mosv_counts\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mx\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m      6\u001B[0m     \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      7\u001B[0m         \u001B[1;32mreturn\u001B[0m \u001B[1;36m0\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001B[0m in \u001B[0;36m__getitem__\u001B[1;34m(self, key)\u001B[0m\n\u001B[0;32m    851\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    852\u001B[0m         \u001B[1;32melif\u001B[0m \u001B[0mkey_is_scalar\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 853\u001B[1;33m             \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_get_value\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    854\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    855\u001B[0m         \u001B[1;32mif\u001B[0m \u001B[0mis_hashable\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001B[0m in \u001B[0;36m_get_value\u001B[1;34m(self, label, takeable)\u001B[0m\n\u001B[0;32m    959\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    960\u001B[0m         \u001B[1;31m# Similar to Index.get_value, but we do not fall back to positional\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m--> 961\u001B[1;33m         \u001B[0mloc\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget_loc\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mlabel\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m    962\u001B[0m         \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mindex\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_get_values_for_loc\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mloc\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mlabel\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m    963\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001B[0m in \u001B[0;36mget_loc\u001B[1;34m(self, key, method, tolerance)\u001B[0m\n\u001B[0;32m   3076\u001B[0m                     \u001B[1;34m\"backfill or nearest lookups\"\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   3077\u001B[0m                 )\n\u001B[1;32m-> 3078\u001B[1;33m             \u001B[0mcasted_key\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_maybe_cast_indexer\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   3079\u001B[0m             \u001B[1;32mtry\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   3080\u001B[0m                 \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_engine\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget_loc\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mcasted_key\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001B[0m in \u001B[0;36m_maybe_cast_indexer\u001B[1;34m(self, key)\u001B[0m\n\u001B[0;32m   5290\u001B[0m         \u001B[0mto\u001B[0m \u001B[0man\u001B[0m \u001B[0mint\u001B[0m \u001B[1;32mif\u001B[0m \u001B[0mequivalent\u001B[0m\u001B[1;33m.\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   5291\u001B[0m         \"\"\"\n\u001B[1;32m-> 5292\u001B[1;33m         \u001B[1;32mif\u001B[0m \u001B[1;32mnot\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mis_floating\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   5293\u001B[0m             \u001B[1;32mreturn\u001B[0m \u001B[0mcom\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mcast_scalar_indexer\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mkey\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   5294\u001B[0m         \u001B[1;32mreturn\u001B[0m \u001B[0mkey\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py\u001B[0m in \u001B[0;36mis_floating\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m   1915\u001B[0m         \u001B[1;32mFalse\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1916\u001B[0m         \"\"\"\n\u001B[1;32m-> 1917\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0minferred_type\u001B[0m \u001B[1;32min\u001B[0m \u001B[1;33m[\u001B[0m\u001B[1;34m\"floating\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m\"mixed-integer-float\"\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m\"integer-na\"\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m   1918\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m   1919\u001B[0m     \u001B[1;33m@\u001B[0m\u001B[0mfinal\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "source": [
    "osv_counts=data['osv'].value_counts()\n",
    "def f(x):\n",
    "    # 判断是否在关键特征值里，是1，否0\n",
    "    if x in osv_counts:\n",
    "        return osv_counts[x]\n",
    "    else:\n",
    "        return 0\n",
    "data['osv_3']=data['osv'].apply(f,args=())\n",
    "data['osv_3']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import re\n",
    "def f(x):\n",
    "    # print(\"x\",x,type(x))\n",
    "    l_v = 0\n",
    "    m_v = 0\n",
    "    s_v = 0\n",
    "    if isinstance(x,str):\n",
    "        it = re.finditer(r\"\\d[\\d|\\.]?\",x)\n",
    "\n",
    "        for i,match in enumerate(it):\n",
    "            if i==0:\n",
    "                l_v= match.group().replace('.','')\n",
    "            elif i==1:\n",
    "                m_v= match.group().replace('.','')\n",
    "            else:\n",
    "                s_v = match.group()\n",
    "        return l_v,m_v,s_v\n",
    "    else:\n",
    "        return l_v,m_v,s_v\n",
    "data['osv_l_v'],data['osv_m_v'],data['osv_s_v']=data['osv'].apply(f,args=())\n",
    "\n",
    "data['osv_l_v']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-64-b91a27adca67>:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['dev_height'][data.dev_height ==0] = 1.0\n",
      "<ipython-input-64-b91a27adca67>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['dev_width'][data.dev_width ==0] = 1.0\n",
      "<ipython-input-64-b91a27adca67>:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  data['dev_ppi'][data.dev_ppi ==0] = 1.0\n"
     ]
    },
    {
     "data": {
      "text/plain": "0            1.0\n1            1.0\n2          760.0\n3         2214.0\n4         2280.0\n           ...  \n499995    1920.0\n499996    1424.0\n499997    1280.0\n499998     960.0\n499999    2040.0\nName: dev_height, Length: 500000, dtype: float64"
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['dev_height'][data.dev_height ==0] = 1.0\n",
    "data['dev_width'][data.dev_width ==0] = 1.0\n",
    "data['dev_ppi'][data.dev_ppi ==0] = 1.0\n",
    "data['dev_height'] = data['dev_height'].astype('float')\n",
    "data['dev_width'] = data['dev_width'].astype('float')\n",
    "data['dev_ppi'] = data['dev_ppi'].astype('float')\n",
    "data['dev_height']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "outputs": [
    {
     "data": {
      "text/plain": "0         1.000000\n1         1.000000\n2         2.111111\n3         2.050000\n4         2.111111\n            ...   \n499995    1.777778\n499996    1.977778\n499997    1.777778\n499998    1.777778\n499999    1.888889\nName: dev_height_width_rate, Length: 500000, dtype: float64"
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['dev_height_width_rate'] = data['dev_height']/data['dev_width']\n",
    "data['dev_height_width_rate']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "outputs": [
    {
     "data": {
      "text/plain": "0               1.0\n1               1.0\n2          273600.0\n3         2391120.0\n4         2462400.0\n            ...    \n499995    2073600.0\n499996    1025280.0\n499997     921600.0\n499998     518400.0\n499999    2203200.0\nName: dev_area, Length: 500000, dtype: float64"
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['dev_area'] = data['dev_height']*data['dev_width']\n",
    "data['dev_area']\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "outputs": [
    {
     "data": {
      "text/plain": "0               1.0\n1               1.0\n2          273600.0\n3         2391120.0\n4         2462400.0\n            ...    \n499995     691200.0\n499996    1025280.0\n499997     921600.0\n499998     518400.0\n499999     734400.0\nName: dev_area_ppi_rate, Length: 500000, dtype: float64"
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['dev_area_ppi_rate'] = data['dev_area']/data['dev_ppi']\n",
    "data['dev_area_ppi_rate']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "data['dev_area_ppi_rate']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "0          miss\n1          miss\n2          miss\n3          miss\n4         zh-CN\n          ...  \n499995    zh-CN\n499996     miss\n499997     miss\n499998    zh_CN\n499999    zh-CN\nName: lan, Length: 500000, dtype: object"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lan_dict={\n",
    "\"zh-CN\":0,\n",
    "\"zh\":1,\n",
    "\"cn\":2,\n",
    "\"zh_CN\":3,\n",
    "\"Zh-CN\":4,\n",
    "\"zh-cn\":5,\n",
    "\"ZH\":6,\n",
    "\"CN\":7,\n",
    "\"tw\":8,\n",
    "\"en\":9,\n",
    "\"zh_CN_#Hans\":10,\n",
    "\"ko\":11,\n",
    "\"zh-TW\":12,\n",
    "\"zh-HK\":13,\n",
    "\"en-US\":14,\n",
    "\"ja\":15,\n",
    "\"en-GB\":16,\n",
    "\"it\":17,\n",
    "\"TW\":18,\n",
    "\"mi\":19,\n",
    "\"zh-MO\":20,\n",
    "\"miss\":21,\n",
    "}\n",
    "def f(x):\n",
    "    # 判断是否在关键特征值里，是1，否0\n",
    "    return lan_dict[x]\n",
    "\n",
    "data['lan'].fillna('miss', inplace=True)\n",
    "data['lan']"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "0         21\n1         21\n2         21\n3         21\n4          0\n          ..\n499995     0\n499996    21\n499997    21\n499998     3\n499999     0\nName: lan_no, Length: 500000, dtype: int64"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['lan_no']=data['lan'].apply(f,args=())\n",
    "data['lan_no']\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "pycharm-eb9f4c10",
   "language": "python",
   "display_name": "PyCharm (Fake_Click_Classification)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}